# -*- coding: utf-8 -*-
import random
import re
import time
from urlparse import urljoin
import MySQLdb
import requests
import hashlib
from lxml import etree
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
from gcpy_utils.proxy.adsl import get_proxy_list
import requests


def get_html():

    config = {"host": "80.240.20.67",
                    "port": 3306,
                    'user': 'root',
                    "passwd": '123456',
                    "db": 'zhp_blog',
                    "charset": "utf8"}
    category_id = 5
    add_time = int(time.time())
    edit_time = int(time.time())
    edit_user_name = '苹果'
    count = 0

    for i in range(1,18):
        headers = {

            'Connection': 'keep-alive ',

            'Cache-Control': 'max-age=600',

            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36',

            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',

            'Accept-Encoding': 'gzip, deflate, br',

            'Accept-Language': 'zh-CN,zh;q=0.9'}

        url = 'https://www.kuaidu.com.cn/sanwen/suibi/list_23_%s.html' % i
        try:
            html = requests.get(url, headers=headers, proxies={"http": 'http://10.101.30.220:3132',
                                                                      "https": 'http://10.101.30.220:3132'})
        except:
            html=get_html()
        html.encoding = 'utf-8'
        html = html.text
        dom = etree.HTML(html)
        urls = dom.xpath('//ul[@class="list2"]/li/h3/a/@href')
        # print urls
        # if urls == []:
        #     # print '----*******8'
        #     pass
        # else:
        for i in urls:
            html2 = requests.get(i, proxies={"http":'http://10.101.30.220:3132', "https":'http://10.101.30.220:3132'})

            html2.encoding = 'utf-8'
            html2= html2.text
            dom2 = etree.HTML(html2)
            try:
                # print '**********************'
                title = dom2.xpath('//div[@class="g_con"]/h1/text()')[0]
                # print title
                # print '**********************'
                content = dom2.xpath('//div[@class="content"]/p/text()')[0]
                print content
                print '**********************'
                detail = content
                # print '**********************'
                con = MySQLdb.connect(**config)
                # print '**********************'
                con.ping()
                cue = con.cursor()
                try:
                    cue.execute(
                        "insert into zhp_blog.article(title,category_id,add_time,edit_time,edit_user_name,`detail`,count) values(%s,%s,%s,%s,%s,%s,%s)",
                        (title, category_id, add_time, edit_time, edit_user_name, str(detail), count))

                    print title
                    print ("存储成功。。。",)

                    con.commit()
                    print 'insert ---------------------success'
                except Exception as e:
                    print e, "-----------2233"
                    con.rollback()
                else:
                    con.commit()
                cue.close()
                con.close()
                # print content
            except Exception as e:
                print e
                title = None
                content = None
                pass
            # if title:
            #     print '---'
                # detail = content
                # con = MySQLdb.connect(**config)
                # con.ping()
                # cue = con.cursor()
                # try:
                #     cue.execute(
                #         "insert into zhp_blog.article(title,category_id,add_time,edit_time,edit_user_name,`detail`,count) values(%s,%s,%s,%s,%s,%s,%s)",
                #         (title,category_id,add_time,edit_time,edit_user_name,str(detail),count))
                #
                #     print title
                #     print ("存储成功。。。",)
                #
                #     con.commit()
                #     print 'insert ---------------------success'
                # except Exception as e:
                #     print e,"-----------2233"
                #     con.rollback()
                # else:
                #     con.commit()
                # cue.close()
                # con.close()
if __name__ == '__main__':

    get_html()